package com.xinqing.bigdata.ml.vector

import org.apache.spark.mllib.linalg
import org.apache.spark.mllib.linalg.{Matrices, Vectors}
import org.apache.spark.mllib.stat.{MultivariateStatisticalSummary, Statistics}
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}


/**
  * @Author:CHQ
  * @Date:2020 /7/22 9:18
  * @Description
  */
object Example1 {
  def main(args: Array[String]): Unit = {

    //TODO dense浓密的 Vectors向量
    val dense: linalg.Vector = Vectors.dense(Array(1.0, 2.0, 3.0, 4.0))
    //TODO sparse稀疏的
    val sparse: linalg.Vector = Vectors.sparse(4, Array(0, 1, 2, 3), Array(5.0, 6.0, 7.0, 8.0))

    //TODO 获取密集矩阵的第三个元素
    println(dense(2))
    //TODO 获取稀疏矩阵的第四个元素
    println(sparse(3))

    //TODO 创建一个矩阵
    //TODO 1.0  3.0  5.0
    //TODO 2.0  4.0  6.0
    println(Matrices.dense(2, 3, Array(1.0, 2.0, 3.0, 4.0, 5.0, 6.0)))


    val conf = new SparkConf().setMaster("local[*]").setAppName("word count")
    val sc = new SparkContext(conf)
    //TODO 读取本地idea文件
    val houseLine = sc.textFile("D://Code//spark-demo//src//main//resources//housedata")

    //TODO RDD[String]  => RDD[lingalg.Vector] 将向量转换成RDD
    val houseVals: RDD[linalg.Vector] = houseLine.map(x => {
      Vectors.dense(x.split(",").map(_.trim.toDouble))
    })

    //TODO 向量转矩阵 上下两行元素个数要对齐
    //TODO Statistics 统计
    //TODO MultivariateStatisticalSummary 多元统计摘要
    val multivariateStatisticalSummary: MultivariateStatisticalSummary = Statistics.colStats(houseVals)

    //TODO 统计列最小值
    println(multivariateStatisticalSummary.min)


  }
}
